{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f470e378-bda0-4dd7-9675-e0be31402e6a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T08:58:25.959746Z",
     "iopub.status.busy": "2022-06-30T08:58:25.959143Z",
     "iopub.status.idle": "2022-06-30T08:58:26.464608Z",
     "shell.execute_reply": "2022-06-30T08:58:26.463875Z",
     "shell.execute_reply.started": "2022-06-30T08:58:25.959695Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "47a268a7-dec2-4b47-ba0a-eb239d792373",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T08:58:10.919486Z",
     "iopub.status.busy": "2022-06-30T08:58:10.918882Z",
     "iopub.status.idle": "2022-06-30T08:58:11.068595Z",
     "shell.execute_reply": "2022-06-30T08:58:11.067538Z",
     "shell.execute_reply.started": "2022-06-30T08:58:10.919437Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('./疫情微博情绪识别挑战赛公开数据/train.csv', sep='\\t')\n",
    "test_df = pd.read_csv('./疫情微博情绪识别挑战赛公开数据/test.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0a7e9e6d-9fb3-4d19-929b-c2a3d5218876",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:02:11.426178Z",
     "iopub.status.busy": "2022-06-30T09:02:11.425557Z",
     "iopub.status.idle": "2022-06-30T09:02:11.441320Z",
     "shell.execute_reply": "2022-06-30T09:02:11.440685Z",
     "shell.execute_reply.started": "2022-06-30T09:02:11.426129Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>words</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>这是在向世界上所有的母亲宣战！[怒] //@子小亻青loukas妈:听说后一直不想看，耐不住...</td>\n",
       "      <td>0</td>\n",
       "      <td>[这, 是, 在, 向, 世界, 上, 所有, 的, 母亲, 宣战, ！, [, 怒, ],...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>和少奶奶@5棒冰 一起收拾完衣柜，就躺在听她给我讲#步步惊心#的情感纠葛「四和八喜欢她」，「...</td>\n",
       "      <td>0</td>\n",
       "      <td>[和, 少奶奶, @, 5, 棒冰,  , 一起, 收拾, 完, 衣柜, ，, 就, 躺, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label  \\\n",
       "0  这是在向世界上所有的母亲宣战！[怒] //@子小亻青loukas妈:听说后一直不想看，耐不住...      0   \n",
       "1  和少奶奶@5棒冰 一起收拾完衣柜，就躺在听她给我讲#步步惊心#的情感纠葛「四和八喜欢她」，「...      0   \n",
       "\n",
       "                                               words  \n",
       "0  [这, 是, 在, 向, 世界, 上, 所有, 的, 母亲, 宣战, ！, [, 怒, ],...  \n",
       "1  [和, 少奶奶, @, 5, 棒冰,  , 一起, 收拾, 完, 衣柜, ，, 就, 躺, ...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f51eeb31-2eb5-4d40-bf80-eb179a511e6b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:02:42.360776Z",
     "iopub.status.busy": "2022-06-30T09:02:42.360169Z",
     "iopub.status.idle": "2022-06-30T09:02:54.758310Z",
     "shell.execute_reply": "2022-06-30T09:02:54.757741Z",
     "shell.execute_reply.started": "2022-06-30T09:02:42.360726Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_df['words'] = train_df['text'].apply(lambda x:' '.join(jieba.lcut(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7965af0a-a202-45aa-b651-2cf2a35f017e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:02:54.759424Z",
     "iopub.status.busy": "2022-06-30T09:02:54.759205Z",
     "iopub.status.idle": "2022-06-30T09:02:56.855760Z",
     "shell.execute_reply": "2022-06-30T09:02:56.855268Z",
     "shell.execute_reply.started": "2022-06-30T09:02:54.759408Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_df['words'] = test_df['text'].apply(lambda x: ' '.join(jieba.lcut(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "38c1c4ab-3c90-4add-98d7-f24f69956a9f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:02:56.856794Z",
     "iopub.status.busy": "2022-06-30T09:02:56.856620Z",
     "iopub.status.idle": "2022-06-30T09:02:56.859104Z",
     "shell.execute_reply": "2022-06-30T09:02:56.858704Z",
     "shell.execute_reply.started": "2022-06-30T09:02:56.856779Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import make_pipeline # 组合流水线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "71235492-47dd-4b55-a9be-a52642b7e1fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:02:56.859891Z",
     "iopub.status.busy": "2022-06-30T09:02:56.859729Z",
     "iopub.status.idle": "2022-06-30T09:03:03.310117Z",
     "shell.execute_reply": "2022-06-30T09:03:03.309225Z",
     "shell.execute_reply.started": "2022-06-30T09:02:56.859877Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer()),\n",
       "                ('logisticregression', LogisticRegression())])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 训练TFIDF和逻辑回归\n",
    "pipline = make_pipeline(\n",
    "    TfidfVectorizer(),\n",
    "    LogisticRegression()\n",
    ")\n",
    "pipline.fit(\n",
    "    train_df['words'].tolist(),\n",
    "    train_df['label'].tolist()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "56b23842-4513-45d7-8a9d-3a0c2a0bea2b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:05:36.686738Z",
     "iopub.status.busy": "2022-06-30T09:05:36.686148Z",
     "iopub.status.idle": "2022-06-30T09:05:36.846921Z",
     "shell.execute_reply": "2022-06-30T09:05:36.845965Z",
     "shell.execute_reply.started": "2022-06-30T09:05:36.686689Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(\n",
    "    {\n",
    "        'label': pipline.predict(test_df['words'])\n",
    "    }\n",
    ").to_csv('lr_submit.csv', index=None) # 86左右"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0c8e03a6-b418-47d5-b116-2f620271b475",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:10:54.432482Z",
     "iopub.status.busy": "2022-06-30T09:10:54.431891Z",
     "iopub.status.idle": "2022-06-30T09:11:22.559996Z",
     "shell.execute_reply": "2022-06-30T09:11:22.559332Z",
     "shell.execute_reply.started": "2022-06-30T09:10:54.432435Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pip install transformers\n",
    "# transformers bert相关的模型使用和加载\n",
    "from transformers import BertTokenizer\n",
    "# 分词器，词典\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "train_encoding = tokenizer(train_df['text'].tolist(), truncation=True, padding=True, max_length=128)\n",
    "test_encoding = tokenizer(test_df['text'].tolist(), truncation=True, padding=True, max_length=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0d1159a1-a991-49ab-aef8-23e442ad1293",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:12:58.412657Z",
     "iopub.status.busy": "2022-06-30T09:12:58.412054Z",
     "iopub.status.idle": "2022-06-30T09:12:58.419056Z",
     "shell.execute_reply": "2022-06-30T09:12:58.418071Z",
     "shell.execute_reply.started": "2022-06-30T09:12:58.412606Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "import torch\n",
    "\n",
    "# 数据集读取\n",
    "class NewsDataset(Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "    \n",
    "    # 读取单个样本\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(int(self.labels[idx]))\n",
    "        return item\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = NewsDataset(train_encoding, train_df['label'])\n",
    "test_dataset = NewsDataset(test_encoding, [0] * len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6ac87f89-2dcf-49c5-824e-8f96f81f9f0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:12:58.767231Z",
     "iopub.status.busy": "2022-06-30T09:12:58.766700Z",
     "iopub.status.idle": "2022-06-30T09:12:59.442078Z",
     "shell.execute_reply": "2022-06-30T09:12:59.440926Z",
     "shell.execute_reply.started": "2022-06-30T09:12:58.767185Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 精度计算\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "febcda95-e18d-443c-9ef9-c2091063ed4a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:16:39.409603Z",
     "iopub.status.busy": "2022-06-30T09:16:39.409010Z",
     "iopub.status.idle": "2022-06-30T09:16:42.171562Z",
     "shell.execute_reply": "2022-06-30T09:16:42.170478Z",
     "shell.execute_reply.started": "2022-06-30T09:16:39.409554Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-chinese and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, AdamW, get_linear_schedule_with_warmup\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-chinese', num_labels=2)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "# 单个读取到批量读取\n",
    "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=False)\n",
    "\n",
    "# 优化方法\n",
    "optim = AdamW(model.parameters(), lr=2e-5)\n",
    "total_steps = len(train_loader) * 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "905f1a3b-09e5-44b5-9018-6bd5fac9de1e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:16:42.172579Z",
     "iopub.status.busy": "2022-06-30T09:16:42.172381Z",
     "iopub.status.idle": "2022-06-30T09:28:53.592197Z",
     "shell.execute_reply": "2022-06-30T09:28:53.591689Z",
     "shell.execute_reply.started": "2022-06-30T09:16:42.172563Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Epoch: 0 ----------------\n",
      "epoth: 0, iter_num: 100, loss: 0.0293, 2.67%\n",
      "epoth: 0, iter_num: 200, loss: 0.0087, 5.33%\n",
      "epoth: 0, iter_num: 300, loss: 0.1835, 8.00%\n",
      "epoth: 0, iter_num: 400, loss: 0.0722, 10.67%\n",
      "epoth: 0, iter_num: 500, loss: 0.0275, 13.33%\n",
      "epoth: 0, iter_num: 600, loss: 0.0207, 16.00%\n",
      "epoth: 0, iter_num: 700, loss: 0.0315, 18.67%\n",
      "epoth: 0, iter_num: 800, loss: 0.0209, 21.33%\n",
      "epoth: 0, iter_num: 900, loss: 0.4200, 24.00%\n",
      "epoth: 0, iter_num: 1000, loss: 0.1209, 26.67%\n",
      "epoth: 0, iter_num: 1100, loss: 0.0093, 29.33%\n",
      "epoth: 0, iter_num: 1200, loss: 0.0229, 32.00%\n",
      "epoth: 0, iter_num: 1300, loss: 0.0164, 34.67%\n",
      "epoth: 0, iter_num: 1400, loss: 0.1712, 37.33%\n",
      "epoth: 0, iter_num: 1500, loss: 0.0070, 40.00%\n",
      "epoth: 0, iter_num: 1600, loss: 0.3227, 42.67%\n",
      "epoth: 0, iter_num: 1700, loss: 0.2320, 45.33%\n",
      "epoth: 0, iter_num: 1800, loss: 0.0102, 48.00%\n",
      "epoth: 0, iter_num: 1900, loss: 0.0195, 50.67%\n",
      "epoth: 0, iter_num: 2000, loss: 0.4099, 53.33%\n",
      "epoth: 0, iter_num: 2100, loss: 0.0076, 56.00%\n",
      "epoth: 0, iter_num: 2200, loss: 0.0008, 58.67%\n",
      "epoth: 0, iter_num: 2300, loss: 0.0496, 61.33%\n",
      "epoth: 0, iter_num: 2400, loss: 0.2253, 64.00%\n",
      "epoth: 0, iter_num: 2500, loss: 0.0046, 66.67%\n",
      "epoth: 0, iter_num: 2600, loss: 0.0968, 69.33%\n",
      "epoth: 0, iter_num: 2700, loss: 0.0118, 72.00%\n",
      "epoth: 0, iter_num: 2800, loss: 0.0165, 74.67%\n",
      "epoth: 0, iter_num: 2900, loss: 0.3721, 77.33%\n",
      "epoth: 0, iter_num: 3000, loss: 0.2609, 80.00%\n",
      "epoth: 0, iter_num: 3100, loss: 0.2001, 82.67%\n",
      "epoth: 0, iter_num: 3200, loss: 0.3607, 85.33%\n",
      "epoth: 0, iter_num: 3300, loss: 0.1450, 88.00%\n",
      "epoth: 0, iter_num: 3400, loss: 0.0155, 90.67%\n",
      "epoth: 0, iter_num: 3500, loss: 0.0164, 93.33%\n",
      "epoth: 0, iter_num: 3600, loss: 0.3011, 96.00%\n",
      "epoth: 0, iter_num: 3700, loss: 0.0728, 98.67%\n",
      "Epoch: 0, Average training loss: 0.1361\n"
     ]
    }
   ],
   "source": [
    "# 训练函数\n",
    "def train():\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    iter_num = 0\n",
    "    total_iter = len(train_loader)\n",
    "    for batch in train_loader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        \n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        \n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        total_train_loss += loss.item()\n",
    "        \n",
    "        # 反向梯度信息\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        # 参数更新\n",
    "        optim.step()\n",
    "\n",
    "        iter_num += 1\n",
    "        if(iter_num % 100==0):\n",
    "            print(\"epoth: %d, iter_num: %d, loss: %.4f, %.2f%%\" % (epoch, iter_num, loss.item(), iter_num/total_iter*100))\n",
    "        \n",
    "    print(\"Epoch: %d, Average training loss: %.4f\"%(epoch, total_train_loss/len(train_loader)))\n",
    "\n",
    "for epoch in range(1):\n",
    "    print(\"------------Epoch: %d ----------------\" % epoch)\n",
    "    train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "51f7ce86-42a8-4a6e-8afb-2f6740955ea7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:28:53.593398Z",
     "iopub.status.busy": "2022-06-30T09:28:53.593219Z",
     "iopub.status.idle": "2022-06-30T09:29:32.603991Z",
     "shell.execute_reply": "2022-06-30T09:29:32.603498Z",
     "shell.execute_reply.started": "2022-06-30T09:28:53.593383Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    pred_label = []\n",
    "    for batch in test_dataloader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        \n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        \n",
    "        outputs = model(input_ids, attention_mask=attention_mask)\n",
    "        pred_label += list(outputs.logits.argmax(1).cpu().data.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "0f22e776-4046-44db-9244-b62495dd4883",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-30T09:29:35.410682Z",
     "iopub.status.busy": "2022-06-30T09:29:35.410066Z",
     "iopub.status.idle": "2022-06-30T09:29:35.425800Z",
     "shell.execute_reply": "2022-06-30T09:29:35.425282Z",
     "shell.execute_reply.started": "2022-06-30T09:29:35.410633Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(\n",
    "    {\n",
    "        'label': pred_label\n",
    "    }\n",
    ").to_csv('bert_submit.csv', index=None) # 96左右"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f457f189-2697-4b65-be33-dd6975abf2d1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
